import scrapy
import re
from binggan.items  import BingganItem

class ZlqSpider(scrapy.Spider):
    name = 'zlq'
    # allowed_domains = ['www.hs.com']
    start_urls = ['h']
    def parse(self, response):
        r = response.xpath('/html/body/div[1]/div[1]/div[3]/ul/li/a/@href').getall()
        name = response.xpath('/html/body/div[1]/div[1]/div[3]/ul/li/a/text()').getall()
        for x,y in zip(r,name):
            if y == '首页' or y == '图片 ' or y == '搜索' or y == '小说 ':
                continue
            yield scrapy.Request(url='https://5gsj08.xyz'+x,meta={'name':y},callback=self.fanye,cb_kwargs=False)
    def fanye(self,response):
        num = re.findall('\d+_(.*?).html',response.xpath('/html/body/div[2]/div[4]/ul/li[12]/a/@href').getall()[0])
        for i in range(1,int(num[0])):
            url = response.xpath('/html/body/div[2]/div[4]/ul/li[13]/select/option['+str(i)+']/@value').get()
            yield scrapy.Request(url=response.url+url,meta={'name':response.meta['name']},callback=self.paqu,priority=3)
    def paqu(self,response):
        url = response.xpath('/html/body/div[2]/div[1]/ul/li/p/a/@href').getall()
        name = response.xpath('/html/body/div[2]/div[1]/ul/li/p/a/text()').getall()
        'https://5gsj08.xyz/h/77460/'
        for x,y in zip(url,name):
            yield scrapy.Request(url='https://5gsj08.xyz/'+x,meta={'name':response.meta['name'],'xname':name},callback=self.xfl,priority=5)
    def xfl(self,response):
        jd = BingganItem()
        jd['bigname'] = response.meta['name']#大分类
        jd['name'] =  response.meta['xname'][0] #名字
        jd['xlei'] = '2020/12/12' #详细分类
        jd['jpg'] = response.xpath('//*[@id="purl"]/text()').get() #图片地址
        jd['url'] = response.url #url

        yield jd